{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# This notebook shows how to calculate the mean AUC on the features from the final layer. The pretrained ACS and random ACS examples are presented. Other models are alike."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "import _init_paths\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "import torch\n",
    "from torch.utils.data import DataLoader\n",
    "import torch.nn.functional as F\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The ``converters`` are currently experimental. It may not support operations including (but not limited to) Functions in ``torch.nn.functional`` that involved data dimension\n"
     ]
    }
   ],
   "source": [
    "from unet import UNet as oriUNet\n",
    "from acsconv.converters import ACSConverter, Conv3dConverter, Conv2_5dConverter\n",
    "from poc_dataset import BaseDatasetVoxel\n",
    "from mylib.utils import plot_multi_shapes, plot_multi_voxels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from poc_config import POCVoxelConfig as cfg\n",
    "from poc_config import POCVoxelEnv as env"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class UNet(oriUNet):\n",
    "    def __init__(self, num_classes):\n",
    "        super().__init__(num_classes)\n",
    "\n",
    "    def get_feature(self, x):\n",
    "        x = self.enc1(x)\n",
    "        x1 = x.clone()\n",
    "        x = self.enc2(x)\n",
    "        x = self.dec1(torch.cat([x1, F.interpolate(x, scale_factor=2)], 1))\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "train_data = env.data_train\n",
    "test_data = env.data_test\n",
    "shape_cp = env.shape_checkpoint\n",
    "\n",
    "train_set = BaseDatasetVoxel(train_data, cfg.train_samples)\n",
    "valid_set = None\n",
    "test_set = BaseDatasetVoxel(test_data, 200)\n",
    "\n",
    "\n",
    "train_loader = DataLoader(train_set, batch_size=cfg.train_batch_size, shuffle=True,\n",
    "                            pin_memory=(torch.cuda.is_available()), num_workers=cfg.num_workers)\n",
    "test_loader = DataLoader(test_set, batch_size=cfg.test_batch_size, shuffle=False,\n",
    "                            pin_memory=(torch.cuda.is_available()), num_workers=cfg.num_workers)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|          | 0/10 [00:00<?, ?it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load shape pretrained weights\n",
      " _IncompatibleKeys(missing_keys=['final.weight', 'final.bias'], unexpected_keys=[])\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hxy/anaconda3/lib/python3.7/site-packages/torch/nn/functional.py:2479: UserWarning: Default upsampling behavior when mode=trilinear is changed to align_corners=False since 0.4.0. Please specify align_corners=True if the old behavior is desired. See the documentation of nn.Upsample for details.\n",
      "  \"See the documentation of nn.Upsample for details.\".format(mode))\n",
      "\n",
      " 10%|█         | 1/10 [00:03<00:27,  3.00s/it]\u001b[A\n",
      " 20%|██        | 2/10 [00:03<00:17,  2.24s/it]\u001b[A\n",
      " 30%|███       | 3/10 [00:03<00:11,  1.70s/it]\u001b[A\n",
      " 40%|████      | 4/10 [00:04<00:07,  1.32s/it]\u001b[A\n",
      " 50%|█████     | 5/10 [00:04<00:05,  1.05s/it]\u001b[A\n",
      " 60%|██████    | 6/10 [00:05<00:03,  1.16it/s]\u001b[A\n",
      " 70%|███████   | 7/10 [00:05<00:02,  1.37it/s]\u001b[A\n",
      " 80%|████████  | 8/10 [00:06<00:01,  1.56it/s]\u001b[A\n",
      " 90%|█████████ | 9/10 [00:06<00:00,  1.75it/s]\u001b[A\n",
      "100%|██████████| 10/10 [00:06<00:00,  1.92it/s]\u001b[A"
     ]
    }
   ],
   "source": [
    "all_features = []\n",
    "gts = []\n",
    "model = UNet(6)\n",
    "model = ACSConverter(model)\n",
    "shape_cp = torch.load(shape_cp)\n",
    "shape_cp.popitem()\n",
    "shape_cp.popitem()\n",
    "incompatible_keys = model.load_state_dict(shape_cp, strict=False)\n",
    "print('load shape pretrained weights\\n', incompatible_keys)\n",
    "model.cuda()\n",
    "model.eval()\n",
    "\n",
    "with torch.no_grad():\n",
    "    for x, y in tqdm(test_loader):\n",
    "        x = x.cuda()\n",
    "        features = F.interpolate(model.get_feature(x), scale_factor=2, mode='trilinear')\n",
    "#         random_features = model_random.get_feature(x)\n",
    "        all_features.append(features.cpu())\n",
    "#         all_random_features.append(random_features.cpu())\n",
    "        del x, features#, random_features\n",
    "        gts.append(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_features = torch.cat(all_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([200, 32, 48, 48, 48])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "gts = torch.cat(gts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "gts = gts[:,:-1].sum(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([200, 48, 48, 48])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gts.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 0.5403825475651183\n",
      "1 0.1285811038302523\n",
      "2 0.7774155076871507\n",
      "3 0.9532965642078928\n",
      "4 0.9406143030116719\n",
      "5 0.8537926228660593\n",
      "6 0.1840238044091505\n",
      "7 0.9724706645354475\n",
      "8 0.9452185636611776\n",
      "9 0.14924687185902605\n",
      "10 0.045249287471890255\n",
      "11 0.8605091591883531\n",
      "12 0.03457575954091149\n",
      "13 0.9500586527417155\n",
      "14 0.9138537897772643\n",
      "15 0.07844444744991355\n",
      "16 0.047762273782486236\n",
      "17 0.9855520298148932\n",
      "18 0.10664903688736317\n",
      "19 0.786775742808291\n",
      "20 0.9704054550941865\n",
      "21 0.24990163005356864\n",
      "22 0.7684907248333583\n",
      "23 0.9508337837017716\n",
      "24 0.7480518110381925\n",
      "25 0.6239492561432597\n",
      "26 0.9293165602636778\n",
      "27 0.9657178439768481\n",
      "28 0.9260579480464312\n",
      "29 0.9868196758165062\n",
      "30 0.09795655645225952\n",
      "31 0.9285995335004847\n",
      "0.6375179222505178\n"
     ]
    }
   ],
   "source": [
    "pretrained_auc = np.zeros(32)\n",
    "for i in range(32):\n",
    "    pretrained_auc[i] = roc_auc_score(gts.flatten(), all_features[:,i].flatten())\n",
    "    print(i, pretrained_auc[i])\n",
    "print(pretrained_auc.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AUC of random ACS "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|          | 0/10 [00:00<?, ?it/s]\u001b[A/home/hxy/anaconda3/lib/python3.7/site-packages/torch/nn/functional.py:2479: UserWarning: Default upsampling behavior when mode=trilinear is changed to align_corners=False since 0.4.0. Please specify align_corners=True if the old behavior is desired. See the documentation of nn.Upsample for details.\n",
      "  \"See the documentation of nn.Upsample for details.\".format(mode))\n",
      "\n",
      " 10%|█         | 1/10 [00:00<00:07,  1.18it/s]\u001b[A\n",
      " 20%|██        | 2/10 [00:01<00:05,  1.38it/s]\u001b[A\n",
      " 30%|███       | 3/10 [00:01<00:04,  1.57it/s]\u001b[A\n",
      " 40%|████      | 4/10 [00:02<00:03,  1.75it/s]\u001b[A\n",
      " 50%|█████     | 5/10 [00:02<00:02,  1.92it/s]\u001b[A\n",
      " 60%|██████    | 6/10 [00:02<00:01,  2.06it/s]\u001b[A\n",
      " 70%|███████   | 7/10 [00:03<00:01,  2.17it/s]\u001b[A\n",
      " 80%|████████  | 8/10 [00:03<00:00,  2.24it/s]\u001b[A\n",
      " 90%|█████████ | 9/10 [00:04<00:00,  2.29it/s]\u001b[A\n",
      "100%|██████████| 10/10 [00:04<00:00,  2.31it/s]\u001b[A"
     ]
    }
   ],
   "source": [
    "all_random_features = []\n",
    "gts = []\n",
    "model_random = UNet(6)\n",
    "model_random = ACSConverter(model_random)\n",
    "model_random.cuda()\n",
    "model_random.eval()\n",
    "with torch.no_grad():\n",
    "    for x, y in tqdm(test_loader):\n",
    "        x = x.cuda()\n",
    "#         features = F.interpolate(model.get_feature(x), scale_factor=2, mode='trilinear')\n",
    "        random_features = F.interpolate(model_random.get_feature(x), scale_factor=2, mode='trilinear')\n",
    "#         all_features.append(features.cpu())\n",
    "        all_random_features.append(random_features.cpu())\n",
    "        del x, random_features#, random_features\n",
    "        gts.append(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_random_features = torch.cat(all_random_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([200, 32, 48, 48, 48])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_random_features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "gts = torch.cat(gts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "gts = gts[:,:-1].sum(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([200, 48, 48, 48])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gts.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 0.12779435424731492\n",
      "1 0.7405222960281823\n",
      "2 0.9621561098148652\n",
      "3 0.8597092005306003\n",
      "4 0.9715571771707239\n",
      "5 0.45470094912905445\n",
      "6 0.12078017754523629\n",
      "7 0.9834419200252179\n",
      "8 0.7180263446420385\n",
      "9 0.4920825008928109\n",
      "10 0.5137663942580295\n",
      "11 0.5\n",
      "12 0.5084743238434396\n",
      "13 0.24642037256899282\n",
      "14 0.4605021639084199\n",
      "15 0.9605542469310315\n",
      "16 0.5128062201767885\n",
      "17 0.49568599826194637\n",
      "18 0.5993036462237372\n",
      "19 0.5000125773477443\n",
      "20 0.8525683099644852\n",
      "21 0.9271359466527214\n",
      "22 0.844142721293847\n",
      "23 0.21938087994973143\n",
      "24 0.797958647061781\n",
      "25 0.8824684869152049\n",
      "26 0.49749708426297345\n",
      "27 0.974609424158798\n",
      "28 0.8037984239910225\n",
      "29 0.49064633101417293\n",
      "30 0.5\n",
      "31 0.7282205513364588\n",
      "0.6327101181296053\n"
     ]
    }
   ],
   "source": [
    "random_auc = np.zeros(32)\n",
    "for i in range(32):\n",
    "    random_auc[i] = roc_auc_score(gts.flatten(), all_random_features[:,i].flatten())\n",
    "    print(i, random_auc[i])\n",
    "print(random_auc.mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7fc7827765c0>"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAD8CAYAAABXXhlaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAACydJREFUeJzt3E2IXYd5h/HnX33mA2MrtY0imdoFUexFI4NwDe6iyDFRnRBp4YLdULQQaJOCQwOp00Ih0EW8ibPpRsQmWoTYqROwMQFjFJlQKLKVWEnsiESKoY0qYbXEJkmhipW8XdyTMBnPeK5m7r1zh/f5wTD3nDlX50XMM+eeM2duqgpJvfzBeg8gafYMX2rI8KWGDF9qyPClhgxfasjwpYYMX2poTeEnOZDkR0nOJ3lkUkNJmq6s9s69JJuAHwP3AReAl4GHquqHyz1na7bVdt63qv1JWtn/8b/8qq5kpe02r2EfdwHnq+p1gCRPAgeBZcPfzvv4s9y7hl1Kejen6sRY263lpf4u4KcLli8M6yTNubUc8Zd6OfGO84YkR4GjANt57xp2J2lS1nLEvwDcsmB5N3Bx8UZVdayq9lXVvi1sW8PuJE3KWo74LwN7ktwG/BfwIPDX1/qPPH/xzBpG0LR95IN713sETcGqw6+qq0n+Fnge2AQ8UVWvTWwySVOzliM+VfVN4JsTmkXSjHjnntSQ4UsNGb7UkOFLDRm+1JDhSw0ZvtSQ4UsNGb7UkOFLDRm+1JDhSw0ZvtSQ4UsNGb7UkOFLDRm+1JDhSw0ZvtSQ4UsNGb7UkOFLDRm+1JDhSw0ZvtSQ4UsNGb7UkOFLDRm+1JDhSw0ZvtSQ4UsNGb7UkOFLDRm+1JDhSw2tGH6SJ5JcTvLqgnU7kryQ5Nzw+YbpjilpksY54n8ZOLBo3SPAiaraA5wYliVtECuGX1XfBn62aPVB4Pjw+DhwaMJzSZqi1Z7j31xVlwCGzzdNbiRJ07Z52jtIchQ4CrCd9057d5LGsNoj/htJdgIMny8vt2FVHauqfVW1bwvbVrk7SZO02vCfBQ4Pjw8Dz0xmHEmzMM6v874K/DvwJ0kuJDkCfB64L8k54L5hWdIGseI5flU9tMyX7p3wLJJmxDv3pIYMX2rI8KWGDF9qyPClhgxfasjwpYYMX2rI8KWGDF9qyPClhgxfasjwpYYMX2rI8KWGDF9qyPClhgxfasjwpYYMX2rI8KWGDF9qyPClhgxfasjwpYYMX2rI8KWGDF9qyPClhgxfasjwpYYMX2rI8KWGDF9qyPClhgxfasjwpYZWDD/JLUlOJjmb5LUkDw/rdyR5Icm54fMN0x9X0iSMc8S/Cny6qm4H7gY+meQO4BHgRFXtAU4My5I2gBXDr6pLVfXd4fEvgLPALuAgcHzY7DhwaFpDSpqsazrHT3IrcCdwCri5qi7B6IcDcNMyzzma5HSS029zZW3TSpqIscNP8n7g68Cnqurn4z6vqo5V1b6q2reFbauZUdKEjRV+ki2Mov9KVX1jWP1Gkp3D13cCl6czoqRJG+eqfoDHgbNV9YUFX3oWODw8Pgw8M/nxJE3D5jG2uQf4G+AHSc4M6/4B+DzwtSRHgP8E/mo6I0qatBXDr6p/A7LMl++d7DiSZsE796SGDF9qyPClhgxfasjwpYYMX2rI8KWGDF9qyPClhgxfamice/Wn6iMf3LveI0jteMSXGjJ8qSHDlxoyfKkhw5caMnypIcOXGjJ8qaF1v4FHv+/5i2dW3mgM3hild+MRX2rI8KWGDF9qyPClhry4t44mdSFvnH/bi31ayCO+1JDhSw0ZvtSQ5/gzNM1z+tXs2/P+vjziSw0ZvtSQ4UsNGb7UkOFLDRm+1JDhSw2tGH6S7UleSvK9JK8l+dyw/rYkp5KcS/JUkq3TH1fSJIxzxL8C7K+qDwF7gQNJ7gYeBR6rqj3Am8CR6Y0paZJWDL9Gfjksbhk+CtgPPD2sPw4cmsqEkiZurHP8JJuSnAEuAy8APwHeqqqrwyYXgF3TGVHSpI0VflX9uqr2AruBu4Dbl9psqecmOZrkdJLTb3Nl9ZNKmphruqpfVW8BLwJ3A9cn+e0f+ewGLi7znGNVta+q9m1h21pmlTQh41zVvzHJ9cPj9wAfBs4CJ4EHhs0OA89Ma0hJkzXOn+XuBI4n2cToB8XXquq5JD8Enkzyz8ArwONTnFPSBK0YflV9H7hzifWvMzrfl7TBeOee1JDvwDNDi9/xZpbvyOO77Wghj/hSQ4YvNWT4UkOGLzXkxb11tNQFt0ld8PNint6NR3ypIcOXGjJ8qSHP8eeM5+aaBY/4UkOGLzVk+FJDhi815MU9aQbGuTFrlhd2PeJLDRm+1JDhSw15ji9N2Gr/0Grx86Z5zu8RX2rI8KWGDF9qyPClhgxfasjwpYYMX2rI8KWGDF9qyDv3pAlb7dum+9d5kqbK8KWGDF9qyHN8aQbm7W3TPeJLDRm+1NDY4SfZlOSVJM8Ny7clOZXkXJKnkmyd3piSJulajvgPA2cXLD8KPFZVe4A3gSOTHEzS9IwVfpLdwEeBLw3LAfYDTw+bHAcOTWNASZM37hH/i8BngN8Myx8A3qqqq8PyBWDXhGeTNCUrhp/kY8DlqvrOwtVLbFrLPP9oktNJTr/NlVWOKWmSxvk9/j3Ax5PcD2wHrmP0CuD6JJuHo/5u4OJST66qY8AxgOuyY8kfDpJma8UjflV9tqp2V9WtwIPAt6rqE8BJ4IFhs8PAM1ObUtJEreX3+H8P/F2S84zO+R+fzEiSpu2abtmtqheBF4fHrwN3TX4kSdPmnXtSQ4YvNWT4UkOGLzVk+FJDhi81ZPhSQ4YvNWT4UkOGLzVk+FJDhi81ZPhSQ4YvNWT4UkOGLzVk+FJDhi81ZPhSQ4YvNWT4UkOGLzVk+FJDhi81ZPhSQ4YvNWT4UkOGLzVk+FJDqarZ7Sz5b+A/gD8E/mdmO56MjTgzbMy5nXn1/qiqblxpo5mG/7udJqerat/Md7wGG3Fm2JhzO/P0+VJfasjwpYbWK/xj67TftdiIM8PGnNuZp2xdzvElrS9f6ksNzTz8JAeS/CjJ+SSPzHr/40jyRJLLSV5dsG5HkheSnBs+37CeMy6W5JYkJ5OcTfJakoeH9XM7d5LtSV5K8r1h5s8N629LcmqY+akkW9d71sWSbErySpLnhuW5n3mhmYafZBPwL8BfAncADyW5Y5YzjOnLwIFF6x4BTlTVHuDEsDxPrgKfrqrbgbuBTw7/t/M89xVgf1V9CNgLHEhyN/Ao8Ngw85vAkXWccTkPA2cXLG+EmX9n1kf8u4DzVfV6Vf0KeBI4OOMZVlRV3wZ+tmj1QeD48Pg4cGimQ62gqi5V1XeHx79g9E25izmeu0Z+OSxuGT4K2A88Payfq5kBkuwGPgp8aVgOcz7zYrMOfxfw0wXLF4Z1G8HNVXUJRpEBN63zPMtKcitwJ3CKOZ97eMl8BrgMvAD8BHirqq4Om8zj98gXgc8AvxmWP8D8z/x7Zh1+lljnrxUmKMn7ga8Dn6qqn6/3PCupql9X1V5gN6NXhLcvtdlsp1peko8Bl6vqOwtXL7Hp3My8lM0z3t8F4JYFy7uBizOeYbXeSLKzqi4l2cnoCDVXkmxhFP1Xquobw+q5nxugqt5K8iKj6xPXJ9k8HEHn7XvkHuDjSe4HtgPXMXoFMM8zv8Osj/gvA3uGK6BbgQeBZ2c8w2o9CxweHh8GnlnHWd5hOM98HDhbVV9Y8KW5nTvJjUmuHx6/B/gwo2sTJ4EHhs3mauaq+mxV7a6qWxl9/36rqj7BHM+8pKqa6QdwP/BjRudy/zjr/Y8541eBS8DbjF6lHGF0HncCODd83rHecy6a+c8Zvbz8PnBm+Lh/nucG/hR4ZZj5VeCfhvV/DLwEnAf+Fdi23rMuM/9fAM9tpJl/++Gde1JD3rknNWT4UkOGLzVk+FJDhi81ZPhSQ4YvNWT4UkP/D8A5KokBpwX7AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(gts[0,30])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7fc7824108d0>"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAD8CAYAAABXXhlaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADMpJREFUeJzt3V+MXOV5x/Hvs+s/C0QITMC1vLRQyWrhogHJci3Ri8gEiZIocEElUFT5wpJvUokokRKnlSpF6gXchFy0N1agcaUokJJIIBSpsoxRGqkxOIGkgEXsoIRYuJgELEKN/+7TizmkO7OznuPdmdlZPd+PNJp53z1nzgPan9/zvnvmTGQmkmqZWukCJI2fwZcKMvhSQQZfKsjgSwUZfKkggy8VZPClgpYV/Ii4OyJej4hjEbFnWEVJGq1Y6pV7ETEN/AK4CzgOvAg8mJmvLbbPulifM1y1pONJGuwM/8u5PBuDtluzjGNsA45l5hsAEfEEcC+waPBnuIq/jDuXcUhJl3IoD7Tabjmn+puB38xrH2/6JE245Yz4/U4nFswbImI3sBtghiuXcThJw7KcEf84cOO89izwVu9Gmbk3M7dm5ta1rF/G4SQNy3KC/yKwJSJujoh1wAPAM8MpS5cUMfghXcKST/Uz80JE/B3wH8A08Hhmvjq0yiSNzHLm+GTmD4AfDKkWSWPilXtSQcsa8TUGfebrsWbtwN3ywvmeDm+xpv/niC8VZPClggy+VJDBlwoy+FJBBl8qyOBLBRl8qSAv4Jl0fS68WXBxTsv9pI844ksFGXypIIMvFWTwpYJc3FuNhrVwNzU9eJu5i8M5li7b1FULb0U/dcPHu9p55UxXO479qN17L70sSauVwZcKMvhSQc7xq+gzn5+aGXy787kzZ/t0Ou8fh975PMC72zd1tU//UffYfe6ddpF2xJcKMvhSQQZfKsjgSwW5uCdNqLxi4eLr6Y3dY/UHN851tecG33kdcMSXSjL4UkEGXyrIOX4VfS666XtxTov9tPo54ksFGXypIIMvFWTwpYJc3KvMhbuJFqfPLOi76n/menq6x+6pc+3e2xFfKsjgSwUNDH5EPB4RJyPilXl9GyJif0QcbZ6vHW2ZkoapzRz/W8A/A/82r28PcCAzH46IPU37K8MvT6pr7uRvF/Rd++Pu9jU9d9l98/0Lrd574IifmT8E3u3pvhfY17zeB9zX6miSJsJS5/gbM/MEQPN8w/BKkjRqI/9zXkTsBnYDzHDlqA8nqYWljvhvR8QmgOb55GIbZubezNyamVvXMviurpJGb6kj/jPATuDh5vnpoVUkCYC506cX9v3qzUvuk9niE5e0+3Ped4D/Av4sIo5HxC46gb8rIo4CdzVtSavEwBE/Mx9c5Ed3DrkWSWPilXtSQQZfKsjgSwUZfKkggy8VZPClggy+VJDBlwoy+FJBBl8qyOBLBRl8qSCDLxVk8KWCDL5UkF+hJa0mET3tnrG75beiOeJLBRl8qSCDLxVk8KWCXNyTJlXvQh4Q69Z1t6enu9sfthvLHfGlggy+VJDBlwpyji9Nqt6Lc+gzp1/TE+GFywJ9OeJLBRl8qSCDLxVk8KWCXNyTVpPei3qmelfz2q3uOeJLBRl8qSCDLxXkHF9aTS5232Ine3+eC3r6csSXCjL4UkEGXypoYPAj4saIOBgRRyLi1Yh4qOnfEBH7I+Jo83zt6MuVNAxtRvwLwJcy8xZgO/D5iLgV2AMcyMwtwIGmLWlY5i4ueMydO9/9+PBM1yPn5lq99cDgZ+aJzPxp8/r3wBFgM3AvsK/ZbB9w35L+4ySN3WXN8SPiJuB24BCwMTNPQOcfB+CGRfbZHRGHI+Lwec4ur1pJQ9E6+BHxMeB7wBcy8/22+2Xm3szcmplb17J+KTVKGrJWF/BExFo6of92Zn6/6X47IjZl5omI2AScHFWRkhpzLb8ja4A2q/oBPAYcycyvz/vRM8DO5vVO4OmhVCRp5NqM+HcAfwv8d0S83PT9PfAw8N2I2AW8CfzNaEqUNGwDg5+ZP2LxD/neOdxyJI2DV+5JBRl8qSCDLxVk8KWCDL5UkMGXCjL4UkEGXyrI4EsFGXypIIMvFWTwpYIMvlSQwZcKMvhSQQZfKsjgSwUZfKkggy8VZPClggy+VJDBlwoy+FJBBl8qyOBLBRl8qSCDLxVk8KWCDL5UkMGXCjL4UkEGXyrI4EsFGXypIIMvFWTwpYIGBj8iZiLihYj4WUS8GhFfa/pvjohDEXE0Ip6MiHWjL1fSMLQZ8c8COzLzE8BtwN0RsR14BHg0M7cA7wG7RlempGEaGPzs+KBprm0eCewAnmr69wH3jaRCSUPXao4fEdMR8TJwEtgP/BI4lZkXmk2OA5tHU6KkYWsV/My8mJm3AbPANuCWfpv12zcidkfE4Yg4fJ6zS69U0tBc1qp+Zp4Cnge2A9dExJrmR7PAW4vsszczt2bm1rWsX06tkoZkzaANIuJ64HxmnoqIK4BP0VnYOwjcDzwB7ASeHmWhUjlT0wu7ZnoGz+nubeKDdmP5wOADm4B9ETFN5wzhu5n5bES8BjwREf8EvAQ81uqIklbcwOBn5s+B2/v0v0Fnvi9plfHKPamgNqf6klbAgvk8MHX9dV3t7N3mV2vbvfeSq5K0ahl8qSCDLxVk8KWCDL5UkMGXCjL4UkEGXyrIC3ikCTV3ps/H2N/5XXe750M6nDvf6r0d8aWCDL5UkMGXCjL4UkEu7kmTau7iwq7Tpy+5S+Zcq7d2xJcKMvhSQQZfKsjgSwUZfKkggy8VZPClggy+VJDBlwoy+FJBBl8qyOBLBRl8qSCDLxVk8KWCDL5UkMGXCjL4UkEGXyrI4EsFtQ5+RExHxEsR8WzTvjkiDkXE0Yh4MiLWja5MScN0OSP+Q8CRee1HgEczcwvwHrBrmIVJGp1WwY+IWeDTwDebdgA7gKeaTfYB942iQEnD13bE/wbwZeCjm3ZfB5zKzAtN+ziweci1SRqRgcGPiM8AJzPzJ/O7+2yai+y/OyIOR8Th8/T59k9JY9fmm3TuAD4bEfcAM8DVdM4AromINc2oPwu81W/nzNwL7AW4Ojb0/cdB0ngNHPEz86uZOZuZNwEPAM9l5ueAg8D9zWY7gadHVqWkoVrO3/G/AnwxIo7RmfM/NpySJI3aZX1pZmY+DzzfvH4D2Db8kiSNmlfuSQUZfKkggy8VZPClggy+VJDBlwoy+FJBBl8qyOBLBRl8qSCDLxVk8KWCDL5UkMGXCjL4UkGX9Xn8VW9qurs5s37gLnNn+twncO7isCqSFjW98YYFfef/vPuetmc2dH+dxdxzP2713o74UkEGXyrI4EsFGXypoFKLe72LeVPXXzd4p3d+t6Br7vTpYZUkLap3IQ/g2M7uBepP3vpaV/v11z5s9d6O+FJBBl8qyOBLBZWa48ea7v/cvHLmsveRxqX34hxYOKf/1z/+z672tnUftHpvR3ypIIMvFWTwpYIMvlRQZOb4DhbxDvBr4OPAb8d24OFYjTXD6qzbmpfuTzLz+kEbjTX4fzhoxOHM3Dr2Ay/DaqwZVmfd1jx6nupLBRl8qaCVCv7eFTrucqzGmmF11m3NI7Yic3xJK8tTfamgsQc/Iu6OiNcj4lhE7Bn38duIiMcj4mREvDKvb0NE7I+Io83ztStZY6+IuDEiDkbEkYh4NSIeavontu6ImImIFyLiZ03NX2v6b46IQ03NT0bEwovWV1hETEfESxHxbNOe+JrnG2vwI2Ia+Bfgr4FbgQcj4tZx1tDSt4C7e/r2AAcycwtwoGlPkgvAlzLzFmA78Pnm/+0k130W2JGZnwBuA+6OiO3AI8CjTc3vAbtWsMbFPAQcmddeDTX/wbhH/G3Ascx8IzPPAU8A9465hoEy84fAuz3d9wL7mtf7gPvGWtQAmXkiM3/avP49nV/KzUxw3dnx0cfJ1jaPBHYATzX9E1UzQETMAp8Gvtm0gwmvude4g78Z+M289vGmbzXYmJknoBMyYOFNzydERNwE3A4cYsLrbk6ZXwZOAvuBXwKnMvNCs8kk/o58A/gyMNe0r2Pya+4y7uBHnz7/rDBEEfEx4HvAFzLz/ZWuZ5DMvJiZtwGzdM4Ib+m32XirWlxEfAY4mZk/md/dZ9OJqbmfcd9l4jhw47z2LPDWmGtYqrcjYlNmnoiITXRGqIkSEWvphP7bmfn9pnvi6wbIzFMR8Tyd9YlrImJNM4JO2u/IHcBnI+IeYAa4ms4ZwCTXvMC4R/wXgS3NCug64AHgmTHXsFTPADub1zuBp1ewlgWaeeZjwJHM/Pq8H01s3RFxfURc07y+AvgUnbWJg8D9zWYTVXNmfjUzZzPzJjq/v89l5ueY4Jr7ysyxPoB7gF/Qmcv9w7iP37LG7wAngPN0zlJ20ZnHHQCONs8bVrrOnpr/is7p5c+Bl5vHPZNcN/AXwEtNza8A/9j0/ynwAnAM+Hdg/UrXukj9nwSeXU01f/Twyj2pIK/ckwoy+FJBBl8qyOBLBRl8qSCDLxVk8KWCDL5U0P8BjL+o+RL4PiEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(all_random_features[0, 29 ,30])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7fc7822b9630>"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAD8CAYAAABXXhlaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGmlJREFUeJzt3WuMnFd5B/D/8859r/Z6bcexHZKUCAKCJmClqaASCiClAZF8oBIXVUGKlC+tFAQIQltVRWql8AX40Io2ahCmQoRLUZNGtChKkyLaKsEhlyYxieM0iR2vvbb3fpn70w87QX7Oc8yM7d3ZWc7/J1n2e3zeec/M7DPvnGfPRVQVRJSWbLMbQET9x8AnShADnyhBDHyiBDHwiRLEwCdKEAOfKEEMfKIEXVLgi8jNIvKiiLwsInevV6OIaGPJxY7cE5EcgJcAfBjAcQC/APBJVX3hfOcUc0NaKYzbwkzMoebsMQC0C/bzqVXyj63BaVkrUic8zkfq5IICibw+wcWk6avEaPAxK21fJ1+1x1nNV5KWfXJaCBsNNIfsxdq+CrKw3f6lR7vgy8LnW1zwL7auVl0ZbbwqllHXWuSdtCI/+j27AcDLqvoKAIjI/QBuBXDewK8UxvH7V95uyrRsf7Ka4xV33soeG+nzV/kvKuGHQWnWXz8MvOqkD+rGmC3TUiTwGvZ1LZ3xUSWRD57GiH3swpJ/fyZ+ZU8ceW3F1cnmls1xc9eYqzP93iFzXB/17SnPBM818qG7fLl/jUoztt6+f59xddrP/cpfkDbc4/pIT/Uu5av+XgDHzjk+3ikjogF3KXf82NcJd3sQkTsB3AkA5by/MxFR/13KHf84gP3nHO8DcCKspKr3quoBVT1QzA2F/01Em+BS7vi/AHCNiFwF4A0AnwDwqd90QmM0j+k/2GXKwiRc1oj0u0fsl4uwHw4ArVJQJv4zrZ23deqTviMuQzZzFf1ak9k+fX3c14rlBFuVIF+Q+dzA8i5blqv5nEelHTxOJEFbnLdlWcO3p7gQ9vF9nWbZP7fSrD1PanV/Ig20iw58VW2KyJ8C+CmAHIBvqerz69YyItowl3LHh6r+BMBP1qktRNQnHLlHlKBLuuNfqMpkFW+/47Ape2l2pzk+/dIOd16uZo8bu32fslCxndh6w3dYCyXbf796Yt7XCZIOUwv+NxG1un3ZZLurgizzv//P5WxZtepHx8yNlM1xY8y/RTt02ByXz/jBMmOvBS9aJFmR1W17woFSAFCe8dcvLAUjeBaW/IPTQOMdnyhBDHyiBDHwiRLEwCdKUF+Te7sL8/jCnp+ash8NHTDH3536PX+i2GZum1h2Vd6yzc7KyUem5w3lbVJwW2HV1Vls2uTaUt1PBawUbSLx8hGfJBzK+xEzS41gslG97OrMVWwb59vjrs7QlE1cFuf853dh1j43WfUJUWna16g94tvTKo24Mtr6eMcnShADnyhBDHyiBPW1j7/SLuLp2n5T9sLCZea4eMoPagkXtdg25PvmH5h80RxfXvArcay0bR/76eUrXJ3Zmp1BGFufaFvZXv892465OuM538Yjq3aC0smcHxxUytnBMQuRhUlq22xZY9S/Zrllm2PIIhNpNBx4M+T7+Iv7I+9HMDZp5xuRPMCpaV9GA4N3fKIEMfCJEsTAJ0oQA58oQX1N7lXbRTy/Ytfj/L9ZOxuvctJPIwtXhmm0/My7sthkVjWyLvRM0yah5hs+cbbatOfFVuAJBwLtKiy4OsNZzZUBNrnXbPvP3SxYukdyPr3olgWPfHxndZsk1GW/Wm97yQ6Eij3X2rauKzVDS8WudWiw8I5PlCAGPlGCGPhECepzHz+PFxd3m7K5GbuazOUn/eSadt72M88sDLs6jy9cbY5fX/LL4tSa9ulWCn4iTRYM2SnkfHvyweo6K23fx11q+cEwry7bfMbJJb+9Tattn2trxb9FWTAWJ1eN7PazYPvvrflFV0eb9vlHFiaOrrzrG9RDHRoofMuIEsTAJ0oQA58oQQx8ogT1NblXypq4euSMKTs6ZhNetbHuG2vm8z6Z1Qg2gF+o+uRavWnrxJJ7EgygqTb9SzQfbGt1puGTdOXInlXFYEP6XGQJ7mojSBTWI4N8gm3GwmWyAUCrdgCRNrpvcyUtP1goXNo8el7TX58GG+/4RAli4BMliIFPlKC+9vH3FJbwF7v+05YV7Qq1/zB1kzsvq9pBLQd2T7k67xp9o+v1w1Vut5f8xJWzNTs4aHZxyNVZyNn8wb5hvxLu7wyd9m0cO2GORwu+A/3cmT3meKnlJ8mECwhLO7ZOUA8keOy6z0uEW2JHRc6jwcY7PlGCGPhECWLgEyWIgU+UoL4m9/LIMJmzybNryzYpN7LPr2ZTq9lVcd4yNOPqTObt7LO9lTlXZ7Fgk3L5cN1u+OW1m00/Pa0e7Gv/4twuV6cdWc/mioptdz5cpxrA0qpNQOYX/WdzfiUYwFNrujpo++fWjTT84xSXIjP/gnyf1Jjc22p4xydKEAOfKEFdA19EviUi0yLy3DllEyLysIgc6fztV70gooHVSx//2wD+FsB3zim7G8AjqnqPiNzdOf5Stweaa2f4l2W70u1zq/vM8bZK1Z3XLtuBLoVI37yqha51Yn16d62gb57L+T5ua9W+bCdO+s+9mSU/8Gd2py2bq0W2xzpp64xFVh0uz9i+eLboXzPtZVCNBp31SB8/vxqZgBOO6Wlzks5W0/WOr6o/AxBm024FcLDz74MAblvndhHRBrrYPv5uVZ0CgM7fPq1NRANrw5N7InKniBwSkUPzM5FfOxFR311s4J8SkT0A0Pn7vHsiq+q9qnpAVQ+MT/R12AARncfFRuKDAG4HcE/n7wd6OamhOZxs2Jls4VZXYyWfqAq3lcpFBr40elgHuq32c26hWXJ1wiW4i0X/LaVZstdq1/y1G3X/0obbczUiW2j1IhxAs17JNa352YLF2e4r98TOo8HWy6/zvgfgfwC8TUSOi8gdWAv4D4vIEQAf7hwT0RbR9Y6vqp88z399cJ3bQkR9wpF7RAnq7yq70sA1pZOmLOy/v1KYdOdVW7aZLfWfV2EfP9bnD/v0Uyt+5Zylul3ldrjk+7iVoh0c02z59gwV/QCaHWW7rVUl7+uc3W0nMa0sjbg6wyftcytHtqmW3IV/puvKqisrHD/b9bx25DwabLzjEyWIgU+UIAY+UYIY+EQJ6mtyTwDkgqld4T7yry36mW6rdTvwZbzoB/ksFWzi7sSqT9ydrdrE2dyq32YrNBpJ7hVydpZfW/0MutjqOs1IUjK0Y8wmAE9s9zP4GkM2uacln8iUkn09JO/fam3Z56F1/1x1cdGVuTqR82iw8Y5PlCAGPlGCGPhECWLgEyWor8m9mhZwtG7X7Hhi/kpzPPXSTndeVrOfT89lfj+3iYrdB+/Y7DZXJ5wxVyz5kXNDQVlsD/ssSFA22j65Vmv7l3apYUfY5dw0O2CoYK8vQ352YKsYzA4s+Wvlhu0SXllkBl17ySYSIZH7QK77rMfoeTTQ+I4RJYiBT5QgBj5Rgvrax29qhjPNUVN2YskOtBk64fuUuWC8zsyeYVcnVAu2uQIAbduBNhIZv5MPBueEswcBQIKy8BgA6i3/PBZX7AXbbT/wJ1zxR1f8W5Rt0NKFsRl9Uu4+yEmWV1yZcletgcY7PlGCGPhECWLgEyWIgU+UoL4m99oQt5z2cjDzrrDgzysE+8EvLPnEXXPUfoZVhvyMsSwYjBNbHquUu/B95Us5n21bhl8Oq7YclC34l78VDFYaPuMTgKWFIAFZ89fXIOHmBuvAz87Lhvx+fzrul/4KyeKSL6z6GZQ0OHjHJ0oQA58oQQx8ogT1tY/f0BxO1cdMWa0R7mvvz2uHXfqcHzCTD/axj62KE5at1H2uIFztJxzQAwCF4FrFSJ1aI7LiTcN+zuar/nM3v2zbmIvsTtUq2jqtIZ9PyEnw/COTbbKKXd1HJidcndXLR11Z1rKvf/FsZJDPQiRZQwODd3yiBDHwiRLEwCdKEAOfKEH9XYGnlcfRBbs3Xr0erCbjF84BgkRVZbvfq22kZLNgp6o+KVUNBtDoauTph3nDkl+BJ1eyybx8wQ+gaUf20wu1yv6xNbPPVXORJGUxeGwtuTrbp+3zz1r+WthuE63V/f7Fn3urTxzmg5d/8lhktuQpX0SDg3d8ogQx8IkSxMAnSlB/J+moYDlYabZYtP3l5f2+vxyuNPv+va+7OrtLdqunQ7jC1ZkWO+GkXvD93nYr6GNHBgKFcjn/OOFKOgBQLNmyViQP0GzYnEet5gfe1Jft26aZr1Oas1uRFSf9ZJvly21uYGWXf5zqpCtC+WxQUOjrjxGtA97xiRLEwCdKEAOfKEFdA19E9ovIoyJyWESeF5G7OuUTIvKwiBzp/O33tyaigdRLVqYJ4POq+ksRGQXwpIg8DOAzAB5R1XtE5G4AdwP40m96oEwUlWCLqOFxu1JOYWLGnffO8Slz/Jnt/+3qDAer6/xr+VpX541J+9m0veBXpZkOZg8+O7vX1Qm3whov+dVmJkp+yendJTtjLR/Znmu5aRNuJyMDkV5fsM/jdMkPvJFgSqM0/UzE5SuCGY0VvyJRtuwTfuUz/KK41XV9B1V1SlV/2fn3IoDDAPYCuBXAwU61gwBu26hGEtH6uqCPbhG5EsD1AB4HsFtVp4C1DwcAu85zzp0ickhEDtXn/VBbIuq/ngNfREYA/DOAz6pqz6ssqOq9qnpAVQ8UxyvdTyCiDdfTyAsRKWAt6L+rqj/uFJ8SkT2qOiUiewBMd30cAIXMDtiZLNsVWt8z5gfn3Fg5ao6vLfrVYFtq+6vvKh9zdfYWZs3xzpz//Jor28cuRfarOl0f6Vpnb2nOlV1fedUclzPfp361bkfMHC3udnXKwaq+sdV+loKtycKVfQC4CUn5ef845cgqv8On7HsoK1xRd6vpJasvAO4DcFhVv3bOfz0I4PbOv28H8MD6N4+INkIvd/z3AfhjAP8rIk93yv4MwD0AfiAidwB4HcAfbUwTiWi9dQ18Vf051r6lx3xwfZtDRP3AX8gSJai/06rE7zc/lrcr57y9ZAfrAMDOnP014OG6/7xaVDtAZaXtf4PQUnveG83ugw1vGD7qyqpD9lonGv5xduYXXdl1kYRf6FhjhzlutP0AmkrOJgVHy34N7oXhYJWgaT+AZ9eTwZZib/gBTdlKZFDPoq3XPh1O16NBxzs+UYIY+EQJYuATJaivffzqagHP/2q/KTs8cpk9vswPWBkv2j7+9IqfuNJsB9tkF3zfNLTa8P1eCXIQI0W/3XZ4rZllP6CoFFl596cT3ZeePXzWvh7ziz5X0Q6u31r0z6N0yr61o8f8hKCxZ2x7mq+86q8VaWOsjLYW3vGJEsTAJ0oQA58oQQx8ogT1NblXmlFc8082WdYq2QEqzWGb3AKA03k7Yjhf9+mlrGDrrIz6gS/BojQQv609cnWb3KtWwz21gFzNXn9H09cJt8ICgFfL27vWGWrYxxpuRx472FIsi1w/v2wTooWzfnCOnvGrHVEaeMcnShADnyhBDHyiBDHwiRLU39l5S6uQ/3raFBUKdqnq0pjf402Kfo/2kA7bEW6VCf84CJJpuVm/BLbM21l1rVk/o05rwWy4yN51WdnvWZ+N2RGHOjHu6tQus3Waw/6xwyWzWmWfJKyN2+uXhvzjDM0GS48t9LyUIm1xvOMTJYiBT5QgBj5RgjZ9Y/OsUrYFl+10derbbf+9XfT91VbFfoY1In3jwqIdsTM87fu07bl5c6wNP8su7NNnRT87LtvlN5avXm2f28y1Pg8w9247q7Cyw+chaqv2eoWSb+Nbd50xxy+8ermrc3XNluWmTro69NuJd3yiBDHwiRLEwCdKEAOfKEGbntxz1M80E1/kBRP2whl0AJBfDabjrfjde9vVHvaBiwzYCYUDigBg8Qo7EGnhGt/Gd77tuDm+YnjW1XnmrE3KFTL/OO/dbvcgfGPeDxZqVuxgoZzE9tfr5cWnrYZ3fKIEMfCJEsTAJ0rQpvfx2yt2gEp23A8iyZ+2A10KucjnVS7od0f6q9qwg2NawWCdnrVtrqDtV+BGlvdtXN0VtGmHzyeES3c/e9YPvDl5eJc5zi/75/qdSVunfNK/1bvO2glJEr6GALQVWaaI/f4tj3d8ogQx8IkSxMAnShADnyhBm57c06adWdbaCqvAhLPzwhmGAJqjfuZdfdQmxfJFnzg7uzJsj2f8SkKjr9nP66FTfgBPs2zbWJ7z18qftq91JI1Hv6V4xydKEAOfKEFdA19EyiLyhIg8IyLPi8hXOuVXicjjInJERL4vIt1XxCSigdBLH78G4CZVXRKRAoCfi8i/AfgcgK+r6v0i8vcA7gDwzQ1s68AI+/TZbr9qUG3Sfw5qsFBPs+Zf/rOrwcSZk/5xhqZtn370NT/ZKLdgVwLO5pdcnfaMnQDEwTrp6HrH1zVv/tQUOn8UwE0AftQpPwjgtg1pIRGtu576+CKSE5GnAUwDeBjAUQBzqvpmSv44gL0b00QiWm89Bb6qtlT1OgD7ANwA4NpYtdi5InKniBwSkUMN1GJViKjPLiirr6pzAB4DcCOAbSLyZid1H4AT5znnXlU9oKoHCvC/2yai/uua3BORnQAaqjonIhUAHwLwVQCPAvg4gPsB3A7ggY1s6EDJ7OelRmYLZnX/Bagwb2fRaeSDML9q61RO+ccZPmGnAxbe8Pvct0+fNcfNFb9MN6Wrl6z+HgAHRSSHtW8IP1DVh0TkBQD3i8hfA3gKwH0b2E4iWkddA19VnwVwfaT8Faz194loi+HIPaIEbfokna1Iq/a3E3LGr4Q7FNlWa6K8zRzXh/3nbnHJDqKpTPvfhBRet9tjtc9G+vhV/gaFzo93fKIEMfCJEsTAJ0oQA58oQUzuXQRt2mW624uLrk427V/a4bItqxR9nfy8nWkns35FovbsnD3uZdsv6p/IFmtZ2Q7Wknz30IvNlmyvBu91++LWTeIdnyhBDHyiBDHwiRLEPv7FCFalifXFNDIpJnfG9tdj21Lrol0pp73sH6ddb7gyGhxhfx4Ash0T5lhH/Dbq4bZv2WpkENZ0MHhrefnCGwje8YmSxMAnShADnyhBDHyiBDG5tx4iS1BrzSdmdHa+60O1g6RguMUYba7YwJtsxG57hskJV6e+187MbJX8IJ/8sn2v85GfK8n58y4G7/hECWLgEyWIgU+UIPbxN0h0gkUPK91Gt7GigeH68wD0ij3meHX/qKuzuM+GmkTe5vFXbJ++UPMDtdrr9PPBOz5Rghj4RAli4BMliIFPlCAm9zZKbFAPB+NsfYWiK2qO25l29RE/yCZM5hVWIlusLdqt0bCy6uqs188Q7/hECWLgEyWIgU+UIAY+UYKY3CO6EFlkubS8LSsu+tF1Y0dtoi43s+Tq6Lxdmq215JfV0nrdlV0M3vGJEsTAJ0oQA58oQezjE12IyNLm+YVacOxPy16dMsfNszPr2qwLxTs+UYIY+EQJ6jnwRSQnIk+JyEOd46tE5HEROSIi3xcRP4iZiAbShdzx7wJw+JzjrwL4uqpeA2AWwB3r2TAi2jg9JfdEZB+AjwD4GwCfExEBcBOAT3WqHATwVwC+uQFtJBoY7cigmuyVExd13mbq9Y7/DQBfBNDuHO8AMKeqb84RPA5g7zq3jYg2SNfAF5GPAphW1SfPLY5U9ROM186/U0QOicihBiK7fxJR3/XyVf99AD4mIrcAKAMYw9o3gG0iku/c9fcBiH7fUdV7AdwLAGMyEf1wIKL+6hr4qvplAF8GABH5AIAvqOqnReSHAD4O4H4AtwN4YAPbSTQQtOEnybRm12fiTD9dyu/xv4S1RN/LWOvz37c+TSKijXZBQ3ZV9TEAj3X+/QqAG9a/SUS00ThyjyhBDHyiBDHwiRLEwCdKEAOfKEEMfKIEMfCJEsTAJ0oQA58oQQx8ogQx8IkSxMAnShADnyhBDHyiBDHwiRLEwCdKEAOfKEEMfKIEMfCJEsTAJ0oQA58oQQx8ogQx8IkSxMAnShADnyhBDHyiBDHwiRLEwCdKEAOfKEGiqv27mMhpAK8BmARwpm8XXh9bsc3A1mw323zx3qKqO7tV6mvg//qiIodU9UDfL3wJtmKbga3ZbrZ54/GrPlGCGPhECdqswL93k657KbZim4Gt2W62eYNtSh+fiDYXv+oTJajvgS8iN4vIiyLysojc3e/r90JEviUi0yLy3DllEyLysIgc6fy9fTPbGBKR/SLyqIgcFpHnReSuTvnAtltEyiLyhIg802nzVzrlV4nI4502f19Eipvd1pCI5ETkKRF5qHM88G0+V18DX0RyAP4OwB8CeAeAT4rIO/rZhh59G8DNQdndAB5R1WsAPNI5HiRNAJ9X1WsB3AjgTzqv7SC3uwbgJlX9XQDXAbhZRG4E8FUAX++0eRbAHZvYxvO5C8Dhc463Qpt/rd93/BsAvKyqr6hqHcD9AG7tcxu6UtWfAZgJim8FcLDz74MAbutro7pQ1SlV/WXn34tY+6HciwFut65Z6hwWOn8UwE0AftQpH6g2A4CI7APwEQD/2DkWDHibQ/0O/L0Ajp1zfLxTthXsVtUpYC3IAOza5Pacl4hcCeB6AI9jwNvd+cr8NIBpAA8DOApgTlWbnSqD+DPyDQBfBNDuHO/A4LfZ6HfgS6SMv1ZYRyIyAuCfAXxWVRc2uz3dqGpLVa8DsA9r3wivjVXrb6vOT0Q+CmBaVZ88tzhSdWDaHJPv8/WOA9h/zvE+ACf63IaLdUpE9qjqlIjswdodaqCISAFrQf9dVf1xp3jg2w0AqjonIo9hLT+xTUTynTvooP2MvA/Ax0TkFgBlAGNY+wYwyG12+n3H/wWAazoZ0CKATwB4sM9tuFgPAri98+/bATywiW1xOv3M+wAcVtWvnfNfA9tuEdkpIts6/64A+BDWchOPAvh4p9pAtVlVv6yq+1T1Sqz9/P6Hqn4aA9zmKFXt6x8AtwB4CWt9uT/v9/V7bOP3AEwBaGDtW8odWOvHPQLgSOfvic1uZ9Dm92Pt6+WzAJ7u/LllkNsN4N0Anuq0+TkAf9kpvxrAEwBeBvBDAKXNbut52v8BAA9tpTa/+Ycj94gSxJF7RAli4BMliIFPlCAGPlGCGPhECWLgEyWIgU+UIAY+UYL+H/Q4LY39kqADAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(all_features[0, 29 ,30])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# mean AUC of pretrained ACS and random ACS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8798684990169666"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.maximum(pretrained_auc, 1-pretrained_auc).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7198669423933144"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# This value could vary due to different randomlization seeds.\n",
    "np.maximum(random_auc, 1-random_auc).mean()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
